#!/usr/bin/perl -I /opt/hp/nv/lib/perl
#
# (C) Copyright 2012- Hewlett-Packard Development Company, L.P.
# 
#  $Id: dbmounts.pl 17656 2012-04-24 20:32:41Z chantal.tremblay@hp.com $
#
# This script will configure database mount points in image for either "sliced" or "shared"
# instance and update the image to all nodes. Eventually, for "shared" config this script get
# disks mounted on all nodes. This script can be run multiple times wihout any side-effect
# caused such as introducing duplicate changes.
#

use strict;
use 5.008000;
use warnings;
use POSIX;
use Getopt::Long; 
use File::Basename;

use lib '/opt/hp/nv/lib/perl';
use config ':all';
use instance ':all';

# possible error returns
use constant {
    OK => 0,
    WARN => 1,
    CRITICAL => 2,
    UNKNOWN => 3,
};

# Global variables

# common global variables
my $proceed=1;
my $done=0;

my $myscriptname = "dbmounts";
my $status_file = "$nvpath/instance/status/" . $myscriptname;
my $revision = '$Revision: 18018 $';
$revision =~ s/\$//g;

# script specific globals
my $IMAGEDIR = "/cm/images";
my $RCLOCAL = "/etc/rc.local";
my $FLAGFILE = "/home/dont_delete_me";
my $PERM = 750;
my $GROUP = "seaquest";
my $cmsh_cmd = "/cm/local/apps/cmd/bin/cmsh";


# command line options
my %gv_opts;

# DO NOT DELETE/DO NOT MODIFY
#
# This sub calls module routine to log informational message to log file
# after first prepending program name; timestamp will be added in
# Perl module
#
sub log_msg
{
    my $message = shift;
    my $infomessage;

    $infomessage = basename($0) . ": INFO: ". $message . "\n";
    instance_log_event($infomessage);

}

# DO NOT DELETE/DO NOT MODIFY
#
# This sub calls module routine to log warning message to log file
# after first prepending program name and Warning label; a timestamp 
# will be added in Perl module
#
sub log_warn
{
    my $message = shift;
    my $warnmessage;

    $warnmessage = basename($0) . ": WARNING: ". $message . "\n";
    instance_log_event($warnmessage);

}

# DO NOT DELETE/DO NOT MODIFY
#
# This sub calls module routine to log error message to log file
# after first prepending program name; timestamp will be added in
# Perl module; the program will then exit
#
sub logerr_n_die
{
    my $message = shift;
    my $errormessage;

    $errormessage = basename($0) . ": ERROR: ". $message . "\n";
    print STDERR $errormessage;

    instance_log_event($errormessage);

    exit(CRITICAL);
}


# CHANGE FOR YOUR NEEDS
#
# This sub checks for any requirements that must be met before 
# the configuration is performed.
#
sub pre_req_check
{
    return (OK) if ($gv_opts{dryrun});
    
    logerr_n_die("Script is required to run as root. Can't continue") if($> != 0);

    return(OK);
}

# CHANGE FOR YOUR NEEDS
#
# This sub verifies that the work done is complete and in such a state 
# that it is ready for the next step to be done
#
sub post_check
{
    return(OK);
}

# CHANGE FOR YOUR NEEDS
#
# This sub checks the script status to determine whether 
# the script has already been run successfully.
# The status file could just hold "completed", for example,
# or it could hold a count of how many things were processed last time,
# e.g. number of nodes, or number of disks, etc.  So that a comparison
# can be made to see if processing must be done again.
#
sub get_status(;$)
{
    my $prefix = shift;
    $prefix = '' unless (defined $prefix);
        
    # Keep the open and close of the file along with the line processing

    # if we haven't yet written this file then we surely must proceed
    return($proceed) if (!-e $status_file);

    open(IFH,$status_file) || logerr_n_die("Can't open file $status_file: $!");
    while (<IFH>) # loop through the whole file
    {
        next if (/^\s*$/);
        next if (/^\#/);

        # ADD HERE THE PROCESSING THAT MAKES SENSE FOR YOUR CODE!
        # IN THIS EXAMPLE WE CHECK FOR 'completed'
        if (/^$prefix:completed/) {
            close(IFH);
            return($done);
        }
    }

    close(IFH);
    return($proceed);
}

# CHANGE FOR YOUR NEEDS
#
# This sub sets the script status to be examined
# the next time the script is run.
# The status file could just hold "completed", for example,
# or it could hold a count of how many things were processed last time,
# e.g. number of nodes, or number of disks, etc.  So that a comparison
# can be made to see if processing must be done again.
#
sub set_status($;$)
{
    my ($statusmsg, $prefix) = @_;       

    return if ($gv_opts{dryrun});  # don't set status if test mode

    # Keep the open and close of the file
    open(OFH,">>$status_file") || logerr_n_die("Can't open file $status_file: $!");

    # ADD HERE THE PROCESSING THAT MAKES SENSE FOR YOUR CODE!
    print OFH "$prefix:$statusmsg\n";

    close(OFH);

}

# CHANGE FOR YOUR NEEDS
#
# This sub verifies that the work done is complete and in such a state 
# that it is ready for the next step to be done
#
sub perform_config
{

    # Keep the following checks to determine whether the
    # script is being forced to be re-run, and to check
    # whether this script completed successfully
    my $force_flag = get_force_flag();
    my $status_value = get_status($gv_opts{squser});

    return (OK) if (!$force_flag && ($status_value == $done)); 
        
    my @mount_points = create_mountpoints();    

    # Updating rc.local is only required for "shared" instance 
    my $config = get_inst_config_type();    
    if ($config eq "shared") {
        run_cmd("/bin/touch $FLAGFILE 2>&1"); 
  
        update_rclocal(@mount_points);
    }        

    # Update the image on all nodes
    update_image();

    update_excludelist("excludelistupdate");
    update_excludelist("excludelistgrab");

    if ($config eq "shared") {
        log_msg("Mounting disks on all nodes");
        mount_disks();
    }

    # Keep the following code to set the status to completed
    # and return success.
    set_status("completed", $gv_opts{squser});
    return(OK);
}

# this sub run the command if not for test mode, or print it to stdout
# and return the output
sub run_cmd($)
{
    my $cmd = shift;

    if ($gv_opts{dryrun}) {
        print STDOUT $cmd . "\n";
        return '';
    }
    else {
        my $output = `$cmd`;
        if (WIFEXITED($?) && (WEXITSTATUS($?) != 0)) {
            logerr_n_die("Error when running '$cmd':" . $?);
        } 
        return $output;
    }
}

#
# This sub creates database mount points in the image
#
sub create_mountpoints
{
    log_msg("Creating database mount points in image");
    
    my $sqconfig = get_instance_sqconfig($gv_opts{squser});
    open(IFH, "$sqconfig") || logerr_n_die("Can't open file $sqconfig: $!");
    my @file_content = <IFH>; 
    close(IFH);
    
    # Look for the lines that contains the db mount points 
    my @dblines = grep(/^\s*[^#]+\s+\/database\/.*\/databasem\//, @file_content);  
    logerr_n_die("The contents of $sqconfig are invalid") unless($#dblines > 0); 

    my $config = get_inst_config_type();
    my @images = get_image_names();
    
    my @mount_points = ();
    for (@dblines) {
        if(/^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)$/){
            my $vol = $1;
            my $dbdir = $3;
            my $dbmdir = $4;

            # Mount point directories for "shared" look differently than "sliced"
            # For "sliced", they look like "/database{m}/squsername/$volume"
            # For "shared", they look like "/database{m}/nxuy"
            my $config = get_inst_config_type();
            if($config eq "shared") {
                $vol = "";
                $dbdir =~ s/\/squserN//; 
                $dbmdir =~ s/\/squserN//; 
                                
                push(@mount_points, $dbdir);
                push(@mount_points, $dbmdir);
            }
            
            for my $image(@images) {
                my $cmd = "/bin/mkdir -p $IMAGEDIR/${image}$dbdir/'$vol' $IMAGEDIR/${image}$dbmdir/'$vol' 2>&1";
                
                run_cmd($cmd);            
            }
        }
    }
    for my $image(@images) {
        my $cmd = "/bin/chmod -R $PERM $IMAGEDIR/$image/database $IMAGEDIR/$image/databasem 2>&1";
        run_cmd($cmd);

        $cmd = "/bin/chgrp -R $GROUP $IMAGEDIR/$image/database $IMAGEDIR/$image/databasem 2>&1";
        run_cmd($cmd);
    }
    
    return @mount_points;
}

#
# This sub adds the codes that perform "disk mounting" for the db mount points 
# into rc.local script in image
#
sub update_rclocal
{
    my @mount_points = @_; # mount_points will contain /database/n2u3 /databasem/n2u3 ...
    
    # The begin and end comment lines are used to identify if the codes to be added
    # already exists
    my $begin_cmnt_line = "### Begin disk mounting for shared instance config ###";
    my $end_cmnt_line = "### End disk mounting for shared instance config ###";
    
    my @images = get_image_names();
    for my $image (@images) {
        my $rcfile = "$IMAGEDIR/${image}$RCLOCAL";
        my $rc_nv = "$IMAGEDIR/${image}$RCLOCAL" . ".nv";
        
        #create rc.local.nv containing all the mount commands
        open(OFH, ">$rc_nv") || logerr_n_die("Can't create file $rc_nv: $!");
        print OFH "HOSTNAME=`uname -n`\n"
                . "case \"\$HOSTNAME\" in\n";

        # Construct string of db mount points to be used as command line argument of lunmgr
        # The shell command lines to be added look like:
        # case "$HOSTNAME" in
        #     /opt/hp/nv/bin/lunmgr --mount      /database/n2u1,/databasem/n2u1,/database/n2u2,/databasem/n2u2,...,/database/n2uy,/databasem/n2uy
        #     ;;
        # n003)
        #      /opt/hp/nv/bin/lunmgr --mount /database/n3u1,/databasem/n3u1,/database/n3u2,/databasem/n3u2,...,/database/n3uy,/databasem/n3uy
        #     ;;
        # ...
        my @nodes = get_instance_nodelist($gv_opts{squser});
        for my $node(@nodes) {
            my $noden = $node;   
            $noden =~ s/n0*(.+)/n$1/;  
            
            my @mountlist = grep(/${noden}u\d+/, @mount_points);
            my $mountstr =join(",", @mountlist);
            if($mountstr) {  
                print OFH "$node)\n"
                        . "   /opt/hp/nv/bin/lunmgr --mount $mountstr\n"
                        . "   ;;\n";

            }
        }
        print OFH "esac";

        close(OFH);

        #then call rc.local.nv from rc.local
        
        if(!-e "$rcfile.ORIG") { # backup rc.local
            run_cmd("/bin/cp -p $rcfile $rcfile.ORIG 2>&1");
        }
        run_cmd("/bin/sed '/$begin_cmnt_line/,/$end_cmnt_line/ d' $rcfile.ORIG > $rcfile");
        
        open(OFH, ">>$rcfile") || logerr_n_die("Can't open file $rcfile: $!");
        
        print OFH "$begin_cmnt_line\n"
                . ". $RCLOCAL.nv\n"
                . "$end_cmnt_line";
        close(OFH);

    }
}

#
# This sub updates image to all nodes
#
sub update_image
{
    my @nodes = get_instance_nodelist($gv_opts{squser});
    my $noderange = "$nodes[0]..$nodes[-1]";
    
    if ($gv_opts{dryrun}) {
        print "Update image for $noderange\n";
        return;
    }
    
    run_cmd("$cmsh_cmd -c \"device; imageupdate -w -n $noderange\" 2>&1");
    
    #####################################################################
    # Make sure all nodes have been provisioned before proceeding further
    #####################################################################
    my $sleep_time_initial=10;
    my $sleep_time_prov=8;
    my $max_prov_iterations=10;
    my $sleep_time_imaged=8;
    my $max_noprogress_iterations=450;  # About 1 hour
 
    #
    # First make sure all Bright provisioners have started
    #
    sleep $sleep_time_initial;  # Allow time for Bright provisioners to start

    # Count the provisioners (which means nodes still TO BE provisioned)
    my $cmdawk = "cmsh -c \"softwareimage; provisioningstatus\" | grep \"Nodes waiting to be provisioned\" | awk '{print \$6}'";
    
    chomp(my $outawk = run_cmd($cmdawk));
    $outawk = Trim($outawk);

    my $lv_iterations=0;
    while (($outawk ne "<none>") && ($lv_iterations < $max_prov_iterations))
    {
        sleep $sleep_time_prov;
        $lv_iterations++;
        
        # Again counting the provisioners (which means nodes still TO BE provisioned)
        chomp ($outawk = run_cmd($cmdawk)); 
        $outawk = Trim($outawk);
        
    }#while


    #
    # All provisioners have started, now check until all nodes are imaged
    #
 
    # Get number of nodes being provisioned
    my $cmdinprov = "cmsh -c \"softwareimage; provisioningstatus\" | grep \"Nodes currently being provisioned\" | wc -l";

    chomp (my $outinprov = run_cmd($cmdinprov));

    my $lv_noprogress=0;
    my $completed=0;
    my $previous_outdone = -1;  # Remember number of nodes done in each iteration
    my $outdone=0;
    
    my $cmddone = "cmsh -c \"softwareimage; provisioningstatus\" | grep \"Nodes currently being provisioned\" | awk '{print \$5}' | grep \"<none>\" | wc -l";
    while (($completed == 0) && ($lv_noprogress < $max_noprogress_iterations))
    {
        $outdone = run_cmd($cmddone);

        if ($outdone == $outinprov) {
            # Number of nodes done with provisiong equals number of nodes that were in the process of provisioning: we're done
            $completed = 1;
        }
        else{
            # Some nodes are not done yet, need to wait more
            if ($previous_outdone != $outdone) {
                # There is progress...the number of nodes nodes is changing
                $previous_outdone = $outdone;   # Remember new value for next iteration
                $lv_noprogress=0;               # Reset the no progress counter
            }
            else { 
                # No progress, start counting how long we have no progress
                $lv_noprogress++;        
            }
            log_msg("Waiting for all instance nodes to be imaged; expected number of imaged nodes: $outinprov, actual: $outdone; noprogress iterations: $lv_noprogress");
            sleep $sleep_time_imaged;
        }
    }

    if ($lv_noprogress == $max_noprogress_iterations)  { 
        # We've waited a long time and some nodes are not imaged yet: log error and exit
        logerr_n_die("Imaging of all nodes did not succeed; expected number of imaged nodes: $outinprov, actual: $outdone");
    
    }
} 

#
# This sub adds database directories to exludelists in Bright
#
sub update_excludelist
{
    my $listname = shift;
    if ($gv_opts{dryrun}) {
        print "Update excludelist for $listname\n";
        return;
    }
    log_msg("Updating $listname in Bright");
    
    my @categories = get_categories();

    my $tmp_file="/tmp/dbmnt.$$";
    my $output;
    my @tmf;
    my $config = get_inst_config_type();
    
    foreach my $category(@categories) {
        # Export the existing excludelist to the temp file
        $output = run_cmd("$cmsh_cmd -c \"category;use $category;get $listname\" 2>&1");
        
        my @tmpf = split('\n', $output);
        # Add database directories to the excludelist
        if($config eq "shared") {
            push(@tmpf, "- /database/*");
            push(@tmpf, "- /databasem/*");
  
        }
        else {
            push(@tmpf, "- /database/$gv_opts{squser}/*");
            push(@tmpf, "- /databasem/$gv_opts{squser}/*");
        }
        # In case the addition of database directories is duplicated
        @tmpf=keys %{{ map { $_ => 1 } @tmpf }};
        @tmpf=sort(@tmpf);
        map {$_ .= "\n"} @tmpf;
                
        # Write the updated excludelist to the temp file
        open(OFH, ">$tmp_file") || logerr_n_die("Can't open file $tmp_file: $!");
        print OFH @tmpf;
        close(OFH);

        # Update the actual excludelist from the temp file
        $output = run_cmd("$cmsh_cmd -c \"category;use $category;set $listname $tmp_file;commit\" 2>&1");
        
    }

    run_cmd("/bin/rm -f $tmp_file");
}

#
# This sub executes rc.local script on all nodes to get the disks mounted
#
sub mount_disks
{
    my @nodes = get_instance_nodelist($gv_opts{squser});
    
    for my $node(@nodes) {
        my $cmd = "ssh $node \"/bin/mount|/bin/egrep '/databasem?/n.+u.+'\" 2>/dev/null";
        
        my $output = run_cmd($cmd);
        if ($output) {
            logerr_n_die("Some disks are already mounted with database directories on node $node");
        }

        $cmd = "ssh $node \". $RCLOCAL 2>&1\"";
        run_cmd($cmd);
        
    }
}


#
# usage
#
sub usage
{
print <<"EOF";

Usage: $0 [-h|--help] --squser=squser [--test]

Options:
    -h|--help: Print this help information.
        --squser squser: Instance user name. This is a required option.
EOF

exit(UNKNOWN);
}


### Main ###

GetOptions( \%gv_opts, 
             "squser=s",
             "dryrun|test" ,
             "help|h|?" => \&usage) || usage(); 

log_n_die("Invalid user") if (!defined($gv_opts{squser}));

log_msg("Starting script: revision number $revision");

pre_req_check;

perform_config;

post_check;

# Return success 
exit(OK);
